<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <title>JAVA线上故障排查全套路 | manor blog</title>
  
    <meta name="keywords" content=" 性能分析 ">
  
  <meta name="description" content="JAVA线上故障排查全套路 | manor blog">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta name="description" content="个人简介pgmanor，程序员一名。 喜欢研究技术，涉及GO、JAVA、微服务、大数据领域。目前主要从事微服务。 联系方式 QQ : 2307546059 (pgmanor) 邮箱 :pgmanor@qq.com">
<meta property="og:type" content="website">
<meta property="og:title" content="关于">
<meta property="og:url" content="https://pgmanor.gitee.io/blog/about/index.html">
<meta property="og:site_name" content="manor blog">
<meta property="og:description" content="个人简介pgmanor，程序员一名。 喜欢研究技术，涉及GO、JAVA、微服务、大数据领域。目前主要从事微服务。 联系方式 QQ : 2307546059 (pgmanor) 邮箱 :pgmanor@qq.com">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2020-12-30T16:00:00.000Z">
<meta property="article:modified_time" content="2020-12-15T14:30:19.631Z">
<meta property="article:author" content="pgmanor">
<meta property="article:tag" content="pgmanor">
<meta property="article:tag" content="blog">
<meta name="twitter:card" content="summary">


<link rel="icon" href="/blog/img/pgmanor_blog.svg">

<link href="/blog/css/style.css?v=1.0.1x" rel="stylesheet">

<link href="/blog/css/hl_theme/atom-light.css?v=1.0.1x" rel="stylesheet">

<link href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/4.1.1/animate.min.css" rel="stylesheet">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/js/fontawesome.min.js" rel="stylesheet">

<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script src="/blog/js/jquery.autocomplete.min.js?v=1.0.1x" ></script>
<script src="/blog/js/titleTip.js?v=1.0.1x" ></script>

<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.0.3/highlight.min.js"></script>
<script>
    hljs.initHighlightingOnLoad();
</script>

<script src="https://cdnjs.cloudflare.com/ajax/libs/nprogress/0.2.0/nprogress.min.js"></script>



<script src="/blog/js/iconfont.js?v=1.0.1x" ></script>

<meta name="generator" content="Hexo 4.2.1"></head>
<div style="display: none">
  <input class="theme_disqus_on" value="false">
  <input class="theme_preload_comment" value="false">
  <input class="theme_blog_path" value="/blog">
  <input id="theme_highlight_on" value="true" />
  <input id="theme_code_copy" value="true" />
</div>




<body>
<aside class="nav">
    <div class="nav-left">
        <a href="/blog/" class="avatar_target">
    <img class="avatar" src="/blog/img/pgmanor_blog.svg" />
</a>
<div class="author">
    <span>pgmanor</span>
</div>

<div class="icon">
    
        
        <a title="github" href="https://gitee.com/pgmanor" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-github"></use>
                </svg>
            
        </a>
        
    
        
        <a title="csdn" href="https://blog.csdn.net/u011622109" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-csdn"></use>
                </svg>
            
        </a>
        
    
        
        <a title="email" href="mailto:pgmanor@qq.com" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-email"></use>
                </svg>
            
        </a>
        
    
        
        <a title="qq" href="http://wpa.qq.com/msgrd?v=3&uin=2307546059&site=qq&menu=yes" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-qq"></use>
                </svg>
            
        </a>
        
    
        
        <a title="kugou" href="https://www.kugou.com/" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-kugou"></use>
                </svg>
            
        </a>
        
    
        
        <a title="neteasemusic" href="https://music.163.com/" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-neteasemusic"></use>
                </svg>
            
        </a>
        
    
</div>



<a class="more-menus">更多菜单</a>


<ul>
    <li><div class="all active">全部文章<small>(129)</small></div></li>
    
        
            
            <li><div data-rel="博客"><i class="fold iconfont icon-right"></i>博客<small>(108)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="B---大数据">B---大数据<small>(4)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="C-----并发">C-----并发<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="D--数据库">D--数据库<small>(16)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="D-devops">D-devops<small>(5)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="F-----架构">F-----架构<small>(10)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="I-----面试">I-----面试<small>(2)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="J-----java">J-----java<small>(14)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="M--微服务">M--微服务<small>(6)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="P-python">P-python<small>(9)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="T-----工具">T-----工具<small>(26)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="T-----测试">T-----测试<small>(6)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="W-----前端">W-----前端<small>(4)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="W-window">W-window<small>(3)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="微服务">微服务<small>(2)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="农场"><i class="fold iconfont icon-right"></i>农场<small>(10)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="微服务">微服务<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="算法">算法<small>(8)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="项目">项目<small>(9)</small></div>
                
            </li>
            
        
    
        
            
            <li><div data-rel="加密">加密<small>(2)</small></div>
                
            </li>
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
</ul>
<div class="left-bottom">
    <div class="menus">
    
    
    
    <a class="dynamic-menu " target="_blank"  href="https://www.iworkh.com/">iworkh</a>
    
    
    <a class="dynamic-menu " target="_blank"  href="https://gitee.com/pgmanor">gitee</a>
    
    </div>
    <div><a class="about  hasFriend  site_url"  href="/blog/about">关于</a><a style="width: 50%"  class="friends">友链</a></div>
</div>
<input type="hidden" id="iworkh_site_posts_number" value="129">
<input type="hidden" id="iworkh_site_word_count" value="273k">
<div style="display: none">
    <span id="busuanzi_value_site_uv"></span>
    <span id="busuanzi_value_site_pv"></span>
</div>

    </div>
    <div class="nav-right">
        <div class="friends-area">
    <div class="friends-title">
        友情链接
        <i class="back-title-list"></i>
    </div>
    <div class="friends-content">
        <ul>
            
            <li><a target="_blank" href="https://www.iworkh.com">iworkh</a></li>
            
            <li><a target="_blank" href="https://www.iworkh.com/download/share/">下载地址</a></li>
            
            <li><a target="_blank" href="http://yelog.org/">叶落阁</a></li>
            
        </ul>
    </div>
</div>
        <div class="title-list">
    <form onkeydown="if(event.keyCode === 13){return false;}">
        <input id="local-search-input" class="search" type="text" placeholder="in: 开头全文搜索" />
        <i class="cross"></i>
        <span>
            <label for="tagswitch">T:</label>
            <input id="tagswitch" type="checkbox" style="display: none" />
            <i id="tagsWitchIcon"></i>
        </span>
    </form>
    <div class="tags-list">
    
    <li class="article-tag-list-item">
        <a class="color4">AQS</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">attr</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">aysnc</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">beanmap</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">bigdecimal</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">blog</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">chrome</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">dict</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">dll</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">doc</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">docker</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">domain</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">elementadmin</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">es</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">excel</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">farm</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">fastjson</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">func</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">git</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">github</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">guide</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">hadoop</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">hexo</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">idea</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">initialzr</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">jackson</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">java</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">jboss</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">jdni</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">jetbrains</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">jfinalshell</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">joplin</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">JPA</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">junit</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">jupyter</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">keepavlie</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">kill</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">linux</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">LocalDate</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">lock</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">markdown</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">maven</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">message</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">mock</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">mq</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">mycat</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">mysql</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">nginx</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">notebook</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">postman</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">powermock</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">python</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">redis</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">rest</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">sharding-jdbc</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">spark</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">springboot</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">testng</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">tornado</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">track</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">transactional</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">uuid</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">vuepress</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">web</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">windows</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">zookeeper</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">分库分表</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">分页</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">加密</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">启动</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">工具</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">并发</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">序列化</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">微服务</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">性能分析</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">手册</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">有道云</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">模板</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">百度云盘</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">算法</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">面试</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">项目</a>
    </li>
    
    <div class="clearfix"></div>
</div>

    
    <div id="local-search-result">

    </div>
    
    <nav id="title-list-nav">
        
        <a id="top" class="农场 "
           href="/blog/2020/12/31/farm-manual-summary/"
           data-tag="手册,farm"
           data-author="pgmanor" >
            <span class="post-title" title="手册汇总">手册汇总</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a id="top" class="农场 微服务 "
           href="/blog/2020/12/31/farm-spring-boot-all-resources/"
           data-tag="springboot"
           data-author="pgmanor" >
            <span class="post-title" title="Spring Boot全网优质教程汇总">Spring Boot全网优质教程汇总</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a id="top" class="加密 "
           href="/blog/2021/01/10/secret-farm-mysql/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="mysql密钥">mysql密钥</span>
            <span class="post-date" title="2021-01-10 00:00:00">2021/01/10</span>
        </a>
        
        <a id="top" class="加密 "
           href="/blog/2020/12/31/secret-baidu-pandisk-share/"
           data-tag="百度云盘"
           data-author="pgmanor" >
            <span class="post-title" title="百度云盘分享链接">百度云盘分享链接</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a id="top" class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-jetbrains-idea/"
           data-tag="idea"
           data-author="pgmanor" >
            <span class="post-title" title="idea使用技巧">idea使用技巧</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a id="top" class="博客 T-----工具 "
           href="/blog/2021/03/14/blog-tool-online/"
           data-tag="blog"
           data-author="pgmanor" >
            <span class="post-title" title="常用在线工具">常用在线工具</span>
            <span class="post-date" title="2021-03-14 00:00:00">2021/03/14</span>
        </a>
        
        <a id="top" class="博客 W-----前端 "
           href="/blog/2020/12/31/blog-web/"
           data-tag="web"
           data-author="pgmanor" >
            <span class="post-title" title="前端常用工具">前端常用工具</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/09/12/project-guide-http/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="模拟http请求">模拟http请求</span>
            <span class="post-date" title="2021-09-12 00:00:00">2021/09/12</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/09/12/blog-tool-chrome-disable-cros/"
           data-tag="chrome"
           data-author="pgmanor" >
            <span class="post-title" title="禁用Chrome跨域">禁用Chrome跨域</span>
            <span class="post-date" title="2021-09-12 00:00:00">2021/09/12</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/09/04/project-guide-download-excel/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="vue和hutool工具实现excel下载">vue和hutool工具实现excel下载</span>
            <span class="post-date" title="2021-09-04 00:00:00">2021/09/04</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/09/04/project-guide-param-pass-array/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="axios参数传数组">axios参数传数组</span>
            <span class="post-date" title="2021-09-04 00:00:00">2021/09/04</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/08/15/blog-mysql-backup-cron/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="cron定时器">cron定时器</span>
            <span class="post-date" title="2021-08-15 00:00:00">2021/08/15</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/08/14/blog-mysql-backup-xtrabackup/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="mysql备份-xtrabackup">mysql备份-xtrabackup</span>
            <span class="post-date" title="2021-08-14 00:00:00">2021/08/14</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/08/14/blog-mysql-backup-xtrabackup-scripts/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="mysql备份-xtrabackup-脚本">mysql备份-xtrabackup-脚本</span>
            <span class="post-date" title="2021-08-14 00:00:00">2021/08/14</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/08/08/project-guide-devops-slowquery/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="SQL慢查询">SQL慢查询</span>
            <span class="post-date" title="2021-08-08 00:00:00">2021/08/08</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/08/08/blog-mysql-backup-summary/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="mysql备份">mysql备份</span>
            <span class="post-date" title="2021-08-08 00:00:00">2021/08/08</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/08/08/project-guide-springboot-upgrade/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="springboot升级到2.5.x遇到的问题">springboot升级到2.5.x遇到的问题</span>
            <span class="post-date" title="2021-08-08 00:00:00">2021/08/08</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2021/08/07/blog-fw-es-install/"
           data-tag="es"
           data-author="pgmanor" >
            <span class="post-title" title="ElasticSearch安装和使用">ElasticSearch安装和使用</span>
            <span class="post-date" title="2021-08-07 00:00:00">2021/08/07</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/08/07/project-guide-log-analysis/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="日志分析">日志分析</span>
            <span class="post-date" title="2021-08-07 00:00:00">2021/08/07</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/08/07/project-guide-devops-monitor/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="运维监控">运维监控</span>
            <span class="post-date" title="2021-08-07 00:00:00">2021/08/07</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/08/01/blog-tool-postman/"
           data-tag="postman"
           data-author="pgmanor" >
            <span class="post-title" title="postman工具使用">postman工具使用</span>
            <span class="post-date" title="2021-08-01 00:00:00">2021/08/01</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/08/01/blog-tool-bug-track/"
           data-tag="track"
           data-author="pgmanor" >
            <span class="post-title" title="bug跟踪工具">bug跟踪工具</span>
            <span class="post-date" title="2021-08-01 00:00:00">2021/08/01</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2021/08/01/blog-java-kill/"
           data-tag="kill"
           data-author="pgmanor" >
            <span class="post-title" title="java项目如何优雅关闭程序">java项目如何优雅关闭程序</span>
            <span class="post-date" title="2021-08-01 00:00:00">2021/08/01</span>
        </a>
        
        <a  class="博客 M--微服务 "
           href="/blog/2021/07/25/blog-spring-boot-logback/"
           data-tag="springboot"
           data-author="pgmanor" >
            <span class="post-title" title="springboot logback多环境">springboot logback多环境</span>
            <span class="post-date" title="2021-07-25 00:00:00">2021/07/25</span>
        </a>
        
        <a  class="博客 W-----前端 "
           href="/blog/2021/06/18/blog-vue-element-admin-keepalive/"
           data-tag="keepavlie"
           data-author="pgmanor" >
            <span class="post-title" title="vue中keep alive失效">vue中keep alive失效</span>
            <span class="post-date" title="2021-06-18 00:00:00">2021/06/18</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2021/06/14/blog-java-start/"
           data-tag="启动"
           data-author="pgmanor" >
            <span class="post-title" title="java启动方式">java启动方式</span>
            <span class="post-date" title="2021-06-14 00:00:00">2021/06/14</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/05/31/project-guide-vue-component/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="vue封装组件">vue封装组件</span>
            <span class="post-date" title="2021-05-31 00:00:00">2021/05/31</span>
        </a>
        
        <a  class="项目 "
           href="/blog/2021/05/09/project-guide-vue-full-calendar/"
           data-tag="项目,guide"
           data-author="pgmanor" >
            <span class="post-title" title="vue-full-calendar日历">vue-full-calendar日历</span>
            <span class="post-date" title="2021-05-09 00:00:00">2021/05/09</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2021/05/04/blog-redis-local/"
           data-tag="redis"
           data-author="pgmanor" >
            <span class="post-title" title="嵌入redis和可视化界面">嵌入redis和可视化界面</span>
            <span class="post-date" title="2021-05-04 22:48:35">2021/05/04</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/03/14/blog-tool-doc/"
           data-tag="doc"
           data-author="pgmanor" >
            <span class="post-title" title="文档工具">文档工具</span>
            <span class="post-date" title="2021-03-14 00:00:00">2021/03/14</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/03/14/blog-jpa-anno-generatedValue/"
           data-tag="JPA"
           data-author="pgmanor" >
            <span class="post-title" title="JPA注解@GeneratedValue">JPA注解@GeneratedValue</span>
            <span class="post-date" title="2021-03-14 00:00:00">2021/03/14</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/03/14/blog-tool-chrome/"
           data-tag="chrome"
           data-author="pgmanor" >
            <span class="post-title" title="Chrome调试技巧">Chrome调试技巧</span>
            <span class="post-date" title="2021-03-14 00:00:00">2021/03/14</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/03/14/blog-vuepress-fulltext-search/"
           data-tag="vuepress"
           data-author="pgmanor" >
            <span class="post-title" title="vuepress支持全文搜索">vuepress支持全文搜索</span>
            <span class="post-date" title="2021-03-14 00:00:00">2021/03/14</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2021/03/07/blog-redis-type-string-sds/"
           data-tag="redis"
           data-author="pgmanor" >
            <span class="post-title" title="Redis为什么将String设计成SDS">Redis为什么将String设计成SDS</span>
            <span class="post-date" title="2021-03-07 00:00:00">2021/03/07</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2021/03/07/blog-redis-fast/"
           data-tag="redis"
           data-author="pgmanor" >
            <span class="post-title" title="Redis为什么这么快">Redis为什么这么快</span>
            <span class="post-date" title="2021-03-07 00:00:00">2021/03/07</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2021/03/07/blog-redis-type-six/"
           data-tag="redis"
           data-author="pgmanor" >
            <span class="post-title" title="Redis的6大数据类型">Redis的6大数据类型</span>
            <span class="post-date" title="2021-03-07 00:00:00">2021/03/07</span>
        </a>
        
        <a  class="博客 微服务 "
           href="/blog/2021/03/06/blog-spring-boot-h2/"
           data-tag="springboot"
           data-author="pgmanor" >
            <span class="post-title" title="springboot整合h2">springboot整合h2</span>
            <span class="post-date" title="2021-03-06 00:00:00">2021/03/06</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2021/02/28/blog-java-md5/"
           data-tag="uuid"
           data-author="pgmanor" >
            <span class="post-title" title="MD5加密">MD5加密</span>
            <span class="post-date" title="2021-02-28 00:00:00">2021/02/28</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2021/02/28/blog-java-uuid/"
           data-tag="uuid"
           data-author="pgmanor" >
            <span class="post-title" title="UUID">UUID</span>
            <span class="post-date" title="2021-02-28 00:00:00">2021/02/28</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2021/02/28/blog-java-base64/"
           data-tag="uuid"
           data-author="pgmanor" >
            <span class="post-title" title="base64加解密">base64加解密</span>
            <span class="post-date" title="2021-02-28 00:00:00">2021/02/28</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2021/02/27/blog-java-new8-date-opertaion.md/"
           data-tag="LocalDate"
           data-author="pgmanor" >
            <span class="post-title" title="java8日期操作">java8日期操作</span>
            <span class="post-date" title="2021-02-27 00:00:00">2021/02/27</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/01/24/blog-jpa-entity-manager/"
           data-tag="JPA"
           data-author="pgmanor" >
            <span class="post-title" title="JPA之EntityManager">JPA之EntityManager</span>
            <span class="post-date" title="2021-01-24 00:00:00">2021/01/24</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2021/01/24/blog-jpa-relationship/"
           data-tag="JPA"
           data-author="pgmanor" >
            <span class="post-title" title="JPA表与表关系映射">JPA表与表关系映射</span>
            <span class="post-date" title="2021-01-24 00:00:00">2021/01/24</span>
        </a>
        
        <a  class="博客 W-----前端 "
           href="/blog/2021/01/24/blog-web-CodeMirror/"
           data-tag="web"
           data-author="pgmanor" >
            <span class="post-title" title="CodeMirror">CodeMirror</span>
            <span class="post-date" title="2021-01-24 00:00:00">2021/01/24</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/01/10/blog-vuepress-theme-password/"
           data-tag="vuepress"
           data-author="pgmanor" >
            <span class="post-title" title="vuepress-theme-reco源码密钥修改">vuepress-theme-reco源码密钥修改</span>
            <span class="post-date" title="2021-01-10 00:00:00">2021/01/10</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/01/03/blog-tool-encryption/"
           data-tag="加密"
           data-author="pgmanor" >
            <span class="post-title" title="在线加密工具">在线加密工具</span>
            <span class="post-date" title="2021-01-03 10:08:09">2021/01/03</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/01/02/blog-tool-jboss-password/"
           data-tag="jboss"
           data-author="pgmanor" >
            <span class="post-title" title="jboss数据库密码加解密">jboss数据库密码加解密</span>
            <span class="post-date" title="2021-01-02 00:00:00">2021/01/02</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2021/01/02/blog-tool-excel-hidden-col/"
           data-tag="excel"
           data-author="pgmanor" >
            <span class="post-title" title="excel隐藏空白列">excel隐藏空白列</span>
            <span class="post-date" title="2021-01-02 00:00:00">2021/01/02</span>
        </a>
        
        <a  class="博客 微服务 "
           href="/blog/2021/01/02/blog-spring-boot-run-withparam-windows/"
           data-tag="springboot"
           data-author="pgmanor" >
            <span class="post-title" title="springboot项目多环境运行报错 windows">springboot项目多环境运行报错 windows</span>
            <span class="post-date" title="2021-01-02 00:00:00">2021/01/02</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-sort/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法8-排序">数据结构和算法8-排序</span>
            <span class="post-date" title="2020-12-31 08:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-search/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法7-搜索">数据结构和算法7-搜索</span>
            <span class="post-date" title="2020-12-31 07:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-graph/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法6-非线性-图">数据结构和算法6-非线性-图</span>
            <span class="post-date" title="2020-12-31 06:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-tree/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法5-非线性-树">数据结构和算法5-非线性-树</span>
            <span class="post-date" title="2020-12-31 05:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-linear-queue/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法4-线性-队列">数据结构和算法4-线性-队列</span>
            <span class="post-date" title="2020-12-31 04:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-linear-stack/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法3-线性-栈">数据结构和算法3-线性-栈</span>
            <span class="post-date" title="2020-12-31 03:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-linear-list/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法2-线性-链">数据结构和算法2-线性-链</span>
            <span class="post-date" title="2020-12-31 02:00:00">2020/12/31</span>
        </a>
        
        <a  class="农场 算法 "
           href="/blog/2020/12/31/farm-data-structures-algorithms-info/"
           data-tag="算法"
           data-author="pgmanor" >
            <span class="post-title" title="数据结构和算法1-介绍">数据结构和算法1-介绍</span>
            <span class="post-date" title="2020-12-31 01:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D-devops "
           href="/blog/2020/12/31/blog-devops-linux-jfinalshell/"
           data-tag="jfinalshell"
           data-author="pgmanor" >
            <span class="post-title" title="linux连接工具jfinalshell">linux连接工具jfinalshell</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-python-tool-buildin-func/"
           data-tag="func"
           data-author="pgmanor" >
            <span class="post-title" title="👍python内置函数">👍python内置函数</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-jpa-query-multi-param/"
           data-tag="JPA"
           data-author="pgmanor" >
            <span class="post-title" title="JPA多条件查询">JPA多条件查询</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-jpa-query/"
           data-tag="JPA"
           data-author="pgmanor" >
            <span class="post-title" title="JPA自定义sql">JPA自定义sql</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2020/12/31/blog-fw-mq-message-missing/"
           data-tag="mq,message"
           data-author="pgmanor" >
            <span class="post-title" title="MQ如何保证消息不丢失?">MQ如何保证消息不丢失?</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 B---大数据 "
           href="/blog/2020/12/31/blog-hadoop-install-windows/"
           data-tag="hadoop"
           data-author="pgmanor" >
            <span class="post-title" title="hadoop开发环境搭建">hadoop开发环境搭建</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-mysql-lock-unlock-table/"
           data-tag="lock"
           data-author="pgmanor" >
            <span class="post-title" title="mysql数据库锁查询和释放">mysql数据库锁查询和释放</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-ai-python-spark-env/"
           data-tag="spark"
           data-author="pgmanor" >
            <span class="post-title" title="python spark开发环境搭建">python spark开发环境搭建</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-mysql-windows-zip-install/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="mysql zip在windows上安装">mysql zip在windows上安装</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-python_install/"
           data-tag="python"
           data-author="pgmanor" >
            <span class="post-title" title="python环境安装">python环境安装</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D-devops "
           href="/blog/2020/12/31/blog-devops-docker-introduction/"
           data-tag="docker"
           data-author="pgmanor" >
            <span class="post-title" title="👍docker入门看这一篇就够了">👍docker入门看这一篇就够了</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 I-----面试 "
           href="/blog/2020/12/31/blog-interview-company/"
           data-tag="面试"
           data-author="pgmanor" >
            <span class="post-title" title="IT互联网公司名单">IT互联网公司名单</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-base-domain/"
           data-tag="domain"
           data-author="pgmanor" >
            <span class="post-title" title="VO DTO DO PO你都了解吗?">VO DTO DO PO你都了解吗?</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-base-serializable/"
           data-tag="序列化"
           data-author="pgmanor" >
            <span class="post-title" title="java序列化">java序列化</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-hexo-theme-math/"
           data-tag="hexo"
           data-author="pgmanor" >
            <span class="post-title" title="hexo主题支持MathJax">hexo主题支持MathJax</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 C-----并发 "
           href="/blog/2020/12/31/blog-java-concurrent-aqs/"
           data-tag="AQS,并发"
           data-author="pgmanor" >
            <span class="post-title" title="java并发AQS">java并发AQS</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-java-dao-sharding-db/"
           data-tag="分库分表"
           data-author="pgmanor" >
            <span class="post-title" title="分库分表中间件-开源产品">分库分表中间件-开源产品</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 B---大数据 "
           href="/blog/2020/12/31/blog-spark-install-windows/"
           data-tag="spark"
           data-author="pgmanor" >
            <span class="post-title" title="spark开发环境搭建">spark开发环境搭建</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-jetbrains-crack/"
           data-tag="idea,jetbrains"
           data-author="pgmanor" >
            <span class="post-title" title="jetbrains全家桶破解">jetbrains全家桶破解</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 M--微服务 "
           href="/blog/2020/12/31/blog-spring-boot-multi-profiles/"
           data-tag="springboot"
           data-author="pgmanor" >
            <span class="post-title" title="springboot多环境配置打包">springboot多环境配置打包</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-markdown-math-express/"
           data-tag="markdown"
           data-author="pgmanor" >
            <span class="post-title" title="markdown数学公式">markdown数学公式</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 I-----面试 "
           href="/blog/2020/12/31/blog-interview-java-question/"
           data-tag="面试"
           data-author="pgmanor" >
            <span class="post-title" title="java常见面试题">java常见面试题</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2020/12/31/blog-nginx-fast/"
           data-tag="nginx"
           data-author="pgmanor" >
            <span class="post-title" title="nginx为何这么快?">nginx为何这么快?</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 M--微服务 "
           href="/blog/2020/12/31/spring-initialzr/"
           data-tag="工具,initialzr"
           data-author="pgmanor" >
            <span class="post-title" title="spring initialzr环境编译发布">spring initialzr环境编译发布</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2020/12/31/blog-redis-memory-eliminate/"
           data-tag="redis"
           data-author="pgmanor" >
            <span class="post-title" title="Redis的内存淘汰策略">Redis的内存淘汰策略</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 M--微服务 "
           href="/blog/2020/12/31/blog-spring-pageable/"
           data-tag="分页"
           data-author="pgmanor" >
            <span class="post-title" title="vue+springboot分页交互">vue+springboot分页交互</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-python_base/"
           data-tag="python"
           data-author="pgmanor" >
            <span class="post-title" title="python基本语法">python基本语法</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 M--微服务 "
           href="/blog/2020/12/31/blog-spring-transaction/"
           data-tag="transactional"
           data-author="pgmanor" >
            <span class="post-title" title="Transactional注解">Transactional注解</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----测试 "
           href="/blog/2020/12/31/blog-test-ng/"
           data-tag="testng"
           data-author="pgmanor" >
            <span class="post-title" title="👍TestNG一篇足以">👍TestNG一篇足以</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----测试 "
           href="/blog/2020/12/31/blog-test-mockito/"
           data-tag="mock"
           data-author="pgmanor" >
            <span class="post-title" title="👍mockito测试框架">👍mockito测试框架</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----测试 "
           href="/blog/2020/12/31/blog-test-powermock/"
           data-tag="powermock"
           data-author="pgmanor" >
            <span class="post-title" title="powermock测试框架">powermock测试框架</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-blogs-all/"
           data-tag="blog"
           data-author="pgmanor" >
            <span class="post-title" title="优质博客和文章">优质博客和文章</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 W-window "
           href="/blog/2020/12/31/blog-tool-windows-dev-install/"
           data-tag="windows"
           data-author="pgmanor" >
            <span class="post-title" title="windows开发环境安装">windows开发环境安装</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-base-bigdecimal/"
           data-tag="bigdecimal"
           data-author="pgmanor" >
            <span class="post-title" title="BigDecimal正确使用了吗?">BigDecimal正确使用了吗?</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 W-----前端 "
           href="/blog/2020/12/31/blog-vue-element-admin/"
           data-tag="elementadmin"
           data-author="pgmanor" >
            <span class="post-title" title="vue element admin开发">vue element admin开发</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 M--微服务 "
           href="/blog/2020/12/31/blog-vue-springboot-package-publish/"
           data-tag="springboot"
           data-author="pgmanor" >
            <span class="post-title" title="vue+springboot打包发布">vue+springboot打包发布</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-java-dao-mycat/"
           data-tag="mycat"
           data-author="pgmanor" >
            <span class="post-title" title="分库分表中间件-MyCat">分库分表中间件-MyCat</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-connect-github/"
           data-tag="github"
           data-author="pgmanor" >
            <span class="post-title" title="github无法访问解决办法">github无法访问解决办法</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-install-windows/"
           data-tag="java"
           data-author="pgmanor" >
            <span class="post-title" title="jdk开发环境搭建">jdk开发环境搭建</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-ai-jupyter-notebook/"
           data-tag="jupyter"
           data-author="pgmanor" >
            <span class="post-title" title="jupyter notebook">jupyter notebook</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-chrome-plugin/"
           data-tag="chrome"
           data-author="pgmanor" >
            <span class="post-title" title="chrome插件">chrome插件</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-git-error-name-too-long/"
           data-tag="git"
           data-author="pgmanor" >
            <span class="post-title" title="git文件名太长">git文件名太长</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-git-setting/"
           data-tag="git"
           data-author="pgmanor" >
            <span class="post-title" title="git配置">git配置</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-python_tornado_info/"
           data-tag="tornado"
           data-author="pgmanor" >
            <span class="post-title" title="👍Tornado入门这一篇足以">👍Tornado入门这一篇足以</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-maven/"
           data-tag="maven"
           data-author="pgmanor" >
            <span class="post-title" title="maven常用命令和配置">maven常用命令和配置</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-notebook-joplin/"
           data-tag="joplin"
           data-author="pgmanor" >
            <span class="post-title" title="joplin笔记">joplin笔记</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-notebook/"
           data-tag="notebook"
           data-author="pgmanor" >
            <span class="post-title" title="常见笔记软件">常见笔记软件</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2020/12/31/blog-fw-distributed-lock-redis-zk/"
           data-tag="lock,redis,zookeeper"
           data-author="pgmanor" >
            <span class="post-title" title="分布式锁用redis还是zk?">分布式锁用redis还是zk?</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-java-dao-shardingSphere/"
           data-tag="sharding-jdbc"
           data-author="pgmanor" >
            <span class="post-title" title="分库分表中间件-ShardingSphere">分库分表中间件-ShardingSphere</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----测试 "
           href="/blog/2020/12/31/blog-test-rest-assured-in-action/"
           data-tag="rest"
           data-author="pgmanor" >
            <span class="post-title" title="rest-assured实战">rest-assured实战</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 F-----架构 "
           href="/blog/2020/12/31/blog-microservice-why/"
           data-tag="微服务"
           data-author="pgmanor" >
            <span class="post-title" title="一定要使用微服务？">一定要使用微服务？</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-tool-bean-map/"
           data-tag="beanmap"
           data-author="pgmanor" >
            <span class="post-title" title="工具类--bean和map互转">工具类--bean和map互转</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-mysql-common-sql/"
           data-tag="mysql"
           data-author="pgmanor" >
            <span class="post-title" title="mysql常用sql">mysql常用sql</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-python-tool-buildin-attr/"
           data-tag="attr"
           data-author="pgmanor" >
            <span class="post-title" title="👍python内置属性(魔法方法)">👍python内置属性(魔法方法)</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-python-tool-obj-dict/"
           data-tag="dict"
           data-author="pgmanor" >
            <span class="post-title" title="python对象与dict互转">python对象与dict互转</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-tool-fastjson-bug/"
           data-tag="fastjson"
           data-author="pgmanor" >
            <span class="post-title" title="Fastjson远程代码高危漏洞">Fastjson远程代码高危漏洞</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 B---大数据 "
           href="/blog/2020/12/31/blog-spark-data-skew/"
           data-tag="spark"
           data-author="pgmanor" >
            <span class="post-title" title="spark数据倾斜及解决方案">spark数据倾斜及解决方案</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 B---大数据 "
           href="/blog/2020/12/31/blog-spark-version-3/"
           data-tag="spark"
           data-author="pgmanor" >
            <span class="post-title" title="spark3.0新特性">spark3.0新特性</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 W-window "
           href="/blog/2020/12/31/blog-tool-windows-cmd/"
           data-tag="windows"
           data-author="pgmanor" >
            <span class="post-title" title="windows常用命令和技巧">windows常用命令和技巧</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 W-window "
           href="/blog/2020/12/31/blog-tool-windows-less-dll/"
           data-tag="dll"
           data-author="pgmanor" >
            <span class="post-title" title="windows缺少dll">windows缺少dll</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/31/blog-tool-youdao-config-edit/"
           data-tag="有道云"
           data-author="pgmanor" >
            <span class="post-title" title="有道云配置修改">有道云配置修改</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 D--数据库 "
           href="/blog/2020/12/31/blog-java-jndi/"
           data-tag="jdni"
           data-author="pgmanor" >
            <span class="post-title" title="JNDI">JNDI</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-online-analysis/"
           data-tag="性能分析"
           data-author="pgmanor" >
            <span class="post-title" title="JAVA线上故障排查全套路">JAVA线上故障排查全套路</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 P-python "
           href="/blog/2020/12/31/blog-tornado-sync-to-async/"
           data-tag="aysnc,tornado"
           data-author="pgmanor" >
            <span class="post-title" title="tornado同步转异步">tornado同步转异步</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 J-----java "
           href="/blog/2020/12/31/blog-java-tool-jackson/"
           data-tag="jackson"
           data-author="pgmanor" >
            <span class="post-title" title="工具类--jackson">工具类--jackson</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----测试 "
           href="/blog/2020/12/31/blog-rest-assured-wiki-info/"
           data-tag="rest"
           data-author="pgmanor" >
            <span class="post-title" title="rest-assured wiki翻译">rest-assured wiki翻译</span>
            <span class="post-date" title="2020-12-31 00:00:00">2020/12/31</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/30/blog-template/"
           data-tag="模板"
           data-author="pgmanor" >
            <span class="post-title" title="hexo博客自定义模板">hexo博客自定义模板</span>
            <span class="post-date" title="2020-12-30 00:00:00">2020/12/30</span>
        </a>
        
        <a  class="博客 T-----工具 "
           href="/blog/2020/12/27/blog-hexo-comments-valine/"
           data-tag="hexo"
           data-author="pgmanor" >
            <span class="post-title" title="hexo扩展评论valine">hexo扩展评论valine</span>
            <span class="post-date" title="2020-12-27 08:59:46">2020/12/27</span>
        </a>
        
        <a  class="博客 D-devops "
           href="/blog/2020/08/15/blog-devops-linux-shell-param/"
           data-tag="linux"
           data-author="pgmanor" >
            <span class="post-title" title="shell 脚本参数解析之getopt和getopts">shell 脚本参数解析之getopt和getopts</span>
            <span class="post-date" title="2020-08-15 00:00:00">2020/08/15</span>
        </a>
        
        <a  class="博客 T-----测试 "
           href="/blog/2020/08/08/blog-test-junit4v5/"
           data-tag="junit"
           data-author="pgmanor" >
            <span class="post-title" title="Junit4和Junit5差异">Junit4和Junit5差异</span>
            <span class="post-date" title="2020-08-08 00:00:00">2020/08/08</span>
        </a>
        
        <a  class="博客 D-devops "
           href="/blog/2020/08/01/blog-devops-linux-command/"
           data-tag="linux"
           data-author="pgmanor" >
            <span class="post-title" title="linux常用命令">linux常用命令</span>
            <span class="post-date" title="2020-08-01 00:00:00">2020/08/01</span>
        </a>
        
        <a  class="博客 D-devops "
           href="/blog/2020/08/01/blog-devops-linux-cmd-adv/"
           data-tag="linux"
           data-author="pgmanor" >
            <span class="post-title" title="linux几个高级命令">linux几个高级命令</span>
            <span class="post-date" title="2020-08-01 00:00:00">2020/08/01</span>
        </a>
        
    </nav>
</div>

    </div>
    <div class="hide-list">
        <div class="semicircle" data-title="切换全屏 快捷键 s">
            <div class="brackets first"><</div>
            <div class="brackets">&gt;</div>
        </div>
    </div>
</aside>
<div class="post">
    <div class="pjax">
        <article id="post-blog/blog-java-online-analysis" class="article article-type-post" itemscope itemprop="blogPost">
    
        <h1 class="article-title">JAVA线上故障排查全套路</h1>
    
    <div class="article-meta">
        
        
        <span class="author"><a>pgmanor</a></span>
        
        
        <span class="book">
            
                <a  data-rel="博客">博客</a>/
            
                <a  data-rel="J-----java">J-----java</a>
            
        </span>
        
        
        <span class="tag">
            
            <a class="color5">性能分析</a>
            
        </span>
        
    </div>
    <div class="article-meta">
        
            发布时间 : <time class="date" title='最后更新: 2021-03-14 11:27:47'>2020-12-31 00:00</time>
        
    </div>
    <div class="article-meta">
        
        <span>字数:5.7k</span>
        
        
        <span id="busuanzi_container_page_pv">
            阅读 :<span id="busuanzi_value_page_pv">
                <span class="count-comment">
                    <span class="spinner">
                      <div class="cube1"></div>
                      <div class="cube2"></div>
                    </span>
                </span>
            </span>
        </span>
        
        
        <span class="top-comment" title="跳转至评论区">
            <a href="#comments">
                评论:<span class="count-comment">
                    <span class="spinner">
                      <div class="cube1"></div>
                      <div class="cube2"></div>
                    </span>
                </span>
            </a>
        </span>
        
    </div>
    
    <div class="article-meta">
        <span class="origin">来源:
            <a href="https://fredal.xin/java-error-check" target="_blank">
                
                JAVA线上故障排查全套路
                
            </a>
        </span>
    </div>
    
    <div class="toc-ref">
    
        <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#1-CPU"><span class="toc-text">1.CPU</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-1-jstack分析cpu问题"><span class="toc-text">1-1.jstack分析cpu问题</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#1-2-频繁gc"><span class="toc-text">1-2.频繁gc</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#1-3-上下文切换"><span class="toc-text">1-3.上下文切换</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-磁盘"><span class="toc-text">2.磁盘</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#2-1-df"><span class="toc-text">2-1.df</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-2-iostat"><span class="toc-text">2-2.iostat</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-3-iotop"><span class="toc-text">2-3.iotop</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-4-lsof"><span class="toc-text">2-4.lsof</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3-内存"><span class="toc-text">3.内存</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#3-1-堆内内存"><span class="toc-text">3-1.堆内内存</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#3-1-1-OOM"><span class="toc-text">3-1-1.OOM</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3-1-2-Stack-Overflow"><span class="toc-text">3-1-2.Stack Overflow</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3-1-3-JMAP定位代码内存泄漏"><span class="toc-text">3-1-3.JMAP定位代码内存泄漏</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3-1-4-gc问题和线程"><span class="toc-text">3-1-4.gc问题和线程</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-2-堆外内存"><span class="toc-text">3-2.堆外内存</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-GC问题"><span class="toc-text">4.GC问题</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#4-1-youngGC过频繁"><span class="toc-text">4-1.youngGC过频繁</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#4-2-youngGC耗时过长"><span class="toc-text">4-2.youngGC耗时过长</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#4-3-触发fullGC"><span class="toc-text">4-3.触发fullGC</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-网络"><span class="toc-text">5.网络</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#5-1-超时"><span class="toc-text">5-1.超时</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5-2-TCP队列溢出"><span class="toc-text">5-2.TCP队列溢出</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5-3-RST异常"><span class="toc-text">5-3.RST异常</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5-4-TIME-WAIT和CLOSE-WAIT"><span class="toc-text">5-4.TIME_WAIT和CLOSE_WAIT</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#5-4-1-TIME-WAIT"><span class="toc-text">5-4-1.TIME_WAIT</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#5-4-2-CLOSE-WAIT"><span class="toc-text">5-4-2.CLOSE_WAIT</span></a></li></ol></li></ol></li></ol>
    
<style>
    .left-col .switch-btn,
    .left-col .switch-area {
        display: none;
    }
    .toc-level-4 i,
    .toc-level-4 ol {
        display: none !important;
    }
</style>
</div>
    
    <div class="article-entry" itemprop="articleBody">
      
        <p>线上故障主要会包括<strong>cpu</strong>、<strong>磁盘</strong>、<strong>内存</strong>以及<strong>网络</strong>问题，而大多数故障可能会包含不止一个层面的问题，所以进行排查时候尽量四个方面依次排查一遍。</p>
<p>例如<strong>jstack</strong>、<strong>jmap</strong>等工具也是不囿于一个方面的问题的，基本上出问题就是<strong>df、free、top</strong> 三连，然后依次<strong>jstack、jmap</strong>伺候，具体问题具体分析即可。</p>
<h2 id="1-CPU"><a href="#1-CPU" class="headerlink" title="1.CPU"></a>1.CPU</h2><p>一般来讲我们首先会排查cpu方面的问题。cpu异常往往还是比较好定位的。原因包括业务逻辑问题(死循环)、频繁gc以及上下文切换过多。<br>而最常见的往往是业务逻辑(或者框架逻辑)导致的，可以使用<strong>jstack</strong>来分析对应的堆栈情况。</p>
<h3 id="1-1-jstack分析cpu问题"><a href="#1-1-jstack分析cpu问题" class="headerlink" title="1-1.jstack分析cpu问题"></a>1-1.jstack分析cpu问题</h3><p>我们先用<code>ps</code>命令找到对应进程的pid(如果你有好几个目标进程，可以先用top看一下哪个占用比较高)。接着用<code>top</code>来找到cpu使用率比较高的一些线程</p>
<pre><code class="shell">top -H -p pid</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083804.png" alt="img"></p>
<p>然后将占用最高的pid转换为16进制得到nid</p>
<pre><code class="shell">printf &#39;%x\n&#39; pid</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083806.png" alt="img"></p>
<p>接着直接在jstack中找到相应的堆栈信息</p>
<pre><code class="shell">jstack pid |grep &#39;nid&#39; -C5 –color</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-83807.png" alt="img"></p>
<blockquote>
<p>可以看到我们已经找到了nid为0x42的堆栈信息，接着只要仔细分析一番即可。</p>
</blockquote>
<p>当然更常见的是我们对整个jstack文件进行分析，通常我们会比较关注<code>WAITING</code>和<code>TIMED_WAITING</code>的部分，<code>BLOCKED</code>就不用说了。<br>我们可以使用命令<code>cat jstack.log | grep &quot;java.lang.Thread.State&quot; | sort -nr | uniq -c</code>来对jstack的状态有一个整体的把握，<br>如果WAITING之类的特别多，那么多半是有问题啦。</p>
<pre><code class="shell">cat jstack.log | grep &quot;java.lang.Thread.State&quot; | sort -nr | uniq -c</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083807.png" alt="img"></p>
<h3 id="1-2-频繁gc"><a href="#1-2-频繁gc" class="headerlink" title="1-2.频繁gc"></a>1-2.频繁gc</h3><p>有时候我们可以先确定下gc是不是太频繁，使用<code>jstat</code>命令来对gc分代变化情况进行观察，1000表示采样间隔(ms)，<br><code>S0C/S1C、S0U/S1U、EC/EU、OC/OU、MC/MU</code>分别代表两个Survivor区、Eden区、老年代、元数据区的容量和使用量。<br><code>YGC/YGT、FGC/FGCT、GCT</code>则代表YoungGc、FullGc的耗时和次数以及总耗时。如果看到gc比较频繁，再针对gc方面做进一步分析，具体可以参考一下gc章节的描述。</p>
<pre><code class="shell">jstat -gc pid 1000</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083808.png" alt="img"></p>
<h3 id="1-3-上下文切换"><a href="#1-3-上下文切换" class="headerlink" title="1-3.上下文切换"></a>1-3.上下文切换</h3><p>针对频繁上下文问题，我们可以使用<code>vmstat</code>命令来进行查看</p>
<pre><code class="shell">vmstat 1</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083809.png" alt="img"></p>
<blockquote>
<p>cs(context switch)一列则代表了上下文切换的次数。</p>
</blockquote>
<p>如果我们希望对特定的pid进行监控那么可以使用<code>pidstat</code>命令，cswch和nvcswch表示自愿及非自愿切换。</p>
<pre><code class="shell">pidstat -w pid</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-83810.png" alt="img"></p>
<h2 id="2-磁盘"><a href="#2-磁盘" class="headerlink" title="2.磁盘"></a>2.磁盘</h2><h3 id="2-1-df"><a href="#2-1-df" class="headerlink" title="2-1.df"></a>2-1.df</h3><p>磁盘问题和cpu一样是属于比较基础的。首先是磁盘空间方面，我们直接使用<code>df</code>来查看文件系统状态</p>
<pre><code class="shell">df -hl</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083810.png" alt="img"></p>
<h3 id="2-2-iostat"><a href="#2-2-iostat" class="headerlink" title="2-2.iostat"></a>2-2.iostat</h3><p>更多时候，磁盘问题还是性能上的问题。我们可以通过<code>iostat</code>来进行分析</p>
<pre><code class="shell">iostat -d -k -x</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083811.png" alt="img"></p>
<blockquote>
<ul>
<li>最后一列<code>%util</code>可以看到每块磁盘写入的程度，在统计时间内所有处理IO时间，除以总共统计时间</li>
<li>而<code>rrqpm/s</code>以及<code>wrqm/s</code>分别表示读写速度，一般就能帮助定位到具体哪块磁盘出现问题了。</li>
</ul>
</blockquote>
<h3 id="2-3-iotop"><a href="#2-3-iotop" class="headerlink" title="2-3.iotop"></a>2-3.iotop</h3><p>另外我们还需要知道是哪个进程在进行读写，一般来说开发自己心里有数，或者用iotop命令来进行定位文件读写的来源。<br><img src="/blog/images/blog/java/analysis/2019-11-04-083812.png" alt="img"></p>
<p>不过这边拿到的是<strong>tid</strong>，我们要转换成<strong>pid</strong>，可以通过<code>readlink</code>来找到pid</p>
<pre><code class="shell">readlink -f /proc/*/task/tid/../..</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-83813.png" alt="img"></p>
<p>找到pid之后就可以看这个<strong>进程</strong>具体的读写情况</p>
<pre><code class="shell">cat /proc/pid/io</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083813.png" alt="img"></p>
<h3 id="2-4-lsof"><a href="#2-4-lsof" class="headerlink" title="2-4.lsof"></a>2-4.lsof</h3><p>我们还可以通过lsof命令来确定具体的文件读写情况</p>
<pre><code class="shell">lsof -p pid</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083814.png" alt="img"></p>
<h2 id="3-内存"><a href="#3-内存" class="headerlink" title="3.内存"></a>3.内存</h2><p>内存问题排查起来相对比CPU麻烦一些，场景也比较多。主要包括OOM、GC问题和堆外内存。一般来讲，我们会先用<code>free</code>命令先来检查内存的各种情况。</p>
<pre><code class="shell">free -ht</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083815.png" alt="img"></p>
<h3 id="3-1-堆内内存"><a href="#3-1-堆内内存" class="headerlink" title="3-1.堆内内存"></a>3-1.堆内内存</h3><p>内存问题大多还都是堆内内存问题。主要分为<code>OOM</code>和<code>StackOverflow</code>。</p>
<h4 id="3-1-1-OOM"><a href="#3-1-1-OOM" class="headerlink" title="3-1-1.OOM"></a>3-1-1.OOM</h4><p>JMV中的内存不足，OOM大致可以分为以下几种：</p>
<ol>
<li><strong>Exception in thread “main” java.lang.OutOfMemoryError: unable to create new native thread</strong></li>
</ol>
<p>这个意思是没有足够的内存空间给线程分配java栈，基本上还是线程池代码写的有问题，比如说忘记shutdown，所以说应该首先从代码层面来寻找问题，使用jstack或者jmap。</p>
<p>如果一切都正常，JVM方面可以通过指定<code>Xss</code>来减少单个thread stack的大小。</p>
<p>另外也可以在系统层面，可以通过修改<code>/etc/security/limits.conf</code>里的<code>nofile</code>和<code>nproc</code>来增大os对线程的限制</p>
<pre><code class="shell">cat /etc/security/limits.conf</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-83816.png" alt="img"></p>
<ol start="2">
<li><strong>Exception in thread “main” java.lang.OutOfMemoryError: Java heap space</strong><br>这个意思是堆的内存占用已经达到-Xmx设置的最大值，应该是最常见的OOM错误了。</li>
</ol>
<p>解决思路仍然是先应该在代码中找，怀疑存在内存泄漏，通过jstack和jmap去定位问题。如果说一切都正常，才需要通过调整<code>Xmx</code>的值来扩大内存。</p>
<ol start="3">
<li><strong>Caused by: java.lang.OutOfMemoryError: Meta space</strong><br>这个意思是元数据区的内存占用已经达到<code>XX:MaxMetaspaceSize</code>设置的最大值，排查思路和上面的一致，参数方面可以通过<code>XX:MaxPermSize</code>来进行调整(这里就不说1.8以前的永久代了)。</li>
</ol>
<h4 id="3-1-2-Stack-Overflow"><a href="#3-1-2-Stack-Overflow" class="headerlink" title="3-1-2.Stack Overflow"></a>3-1-2.Stack Overflow</h4><p>栈内存溢出，这个大家见到也比较多。<br><strong>Exception in thread “main” java.lang.StackOverflowError</strong><br>表示线程栈需要的内存大于Xss值，同样也是先进行排查，参数方面通过<code>Xss</code>来调整，但调整的太大可能又会引起OOM。</p>
<h4 id="3-1-3-JMAP定位代码内存泄漏"><a href="#3-1-3-JMAP定位代码内存泄漏" class="headerlink" title="3-1-3.JMAP定位代码内存泄漏"></a>3-1-3.JMAP定位代码内存泄漏</h4><p>上述关于OOM和StackOverflow的代码排查方面，我们一般使用JMAP来导出dump文件</p>
<pre><code class="shell">jmap -dump:format=b,file=filename pid</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083817.png" alt="img"></p>
<p>通过mat(Eclipse Memory Analysis Tools)导入dump文件进行分析，内存泄漏问题一般我们直接选Leak Suspects即可，mat给出了内存泄漏的建议。</p>
<p>另外也可以选择Top Consumers来查看最大对象报告。和线程相关的问题可以选择thread overview进行分析。</p>
<p>除此之外就是选择Histogram类概览来自己慢慢分析，大家可以搜搜mat的相关教程。<a href="https://www.iworkh.com/book/dev/iworkh-dev-jvm/docs/#/zh-cn/3-4-jvm-memory-dump" target="_blank" rel="noopener">iworkh-jvm-实战-内存dump</a></p>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083818.png" alt="img"></p>
<p>日常开发中，代码产生内存泄漏是比较常见的事，并且比较隐蔽，需要开发者更加关注细节。</p>
<blockquote>
<ul>
<li>比如说每次请求都new对象，导致大量重复创建对象；</li>
<li>进行文件流操作但未正确关闭；</li>
<li>手动不当触发gc；</li>
<li>ByteBuffer缓存分配不合理等都会造成代码OOM。</li>
</ul>
</blockquote>
<p>另一方面，我们可以在启动参数中指定<code>-XX:+HeapDumpOnOutOfMemoryError</code>来保存OOM时的dump文件。</p>
<h4 id="3-1-4-gc问题和线程"><a href="#3-1-4-gc问题和线程" class="headerlink" title="3-1-4.gc问题和线程"></a>3-1-4.gc问题和线程</h4><p>gc问题除了影响cpu也会影响内存，排查思路也是一致的。一般先使用jstat来查看分代变化情况，比如youngGC或者fullGC次数是不是太多呀；EU、OU等指标增长是不是异常呀等。</p>
<p>线程的话太多而且不被及时gc也会引发oom，大部分就是之前说的<code>unable to create new native thread</code>。</p>
<p>除了jstack细细分析dump文件外，我们一般先会看下总体线程，通过<code>pstree</code>。</p>
<pre><code class="shell">pstree -p pid |wc -l</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083819.png" alt="img"></p>
<p>或者直接通过查看<code>/proc/pid/task</code>的数量即为线程数量。</p>
<pre><code class="shell">ls -l /proc/pid/task |wc -l</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083820.png" alt="img"></p>
<h3 id="3-2-堆外内存"><a href="#3-2-堆外内存" class="headerlink" title="3-2.堆外内存"></a>3-2.堆外内存</h3><p>如果碰到堆外内存溢出，那可真是太不幸了。首先堆外内存溢出表现就是物理常驻内存增长快，报错的话使用方式都不确定。</p>
<ul>
<li>如果由于使用Netty导致的，那错误日志里可能会出现<code>OutOfDirectMemoryError</code>错误，</li>
<li>如果直接是DirectByteBuffer，那会报<code>OutOfMemoryError: Direct buffer memory</code>。</li>
</ul>
<p>堆外内存溢出往往是和NIO的使用相关，一般我们先通过<code>pmap</code>来查看下进程占用的内存情况，</p>
<pre><code class="shell">pmap -x pid | sort -rn -k3 | head -30</code></pre>
<blockquote>
<p>查看对应pid倒序前30大的内存段。<br>可再一段时间后再跑一次命令看看内存增长情况，或者和正常机器比较可疑的内存段在哪里。</p>
</blockquote>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-83821.png" alt="img"></p>
<p>我们如果确定有可疑的内存端，需要通过<code>gdb</code>来分析</p>
<pre><code class="shell">gdb --batch --pid {pid} -ex &quot;dump memory filename.dump {内存起始地址} {内存起始地址+内存块大小}&quot;</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083821.png" alt="img"></p>
<p>获取dump文件后可用<code>heaxdump</code>进行查看，不过大多数看到的都是二进制乱码。</p>
<pre><code class="shell">hexdump -C filename | less</code></pre>
<p>NMT是Java7U40引入的HotSpot新特性，配合<code>jcmd</code>命令我们就可以看到具体内存组成了。</p>
<p>需要在启动参数中加入 <code>-XX:NativeMemoryTracking=summary</code> 或者 <code>-XX:NativeMemoryTracking=detail</code>，会有略微性能损耗。</p>
<p>一般对于堆外内存缓慢增长直到爆炸的情况来说，可以先设一个基线。</p>
<pre><code class="shell">jcmd pid VM.native_memory baseline</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083822.png" alt="img"></p>
<p>然后等放一段时间后再去看看内存增长的情况，通过<code>jcmd</code>做一下summary或者detail级别的diff。</p>
<pre><code class="shell">jcmd pid VM.native_memory detail.diff(summary.diff)</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083823.png" alt="img"></p>
<img src="/blog/images/blog/java/analysis/2019-11-04-83824.png" width="650px">

<p>可以看到<code>jcmd</code>分析出来的内存十分详细，包括<strong>堆内、线程以及gc</strong>(所以上述其他内存异常其实都可以用nmt来分析)，这边堆外内存我们重点关注Internal的内存增长，如果增长十分明显的话那就是有问题了。</p>
<p>detail级别的话还会有具体内存段的增长情况，如下图。<br><img src="/blog/images/blog/java/analysis/2019-11-04-083824.png" alt="img"></p>
<p>此外在系统层面，我们还可以使用strace命令来监控内存分配 <code>strace -f -e &quot;brk,mmap,munmap&quot; -p pid</code><br>这边内存分配信息主要包括了pid和内存地址。<br><img src="/blog/images/blog/java/analysis/2019-11-04-083825.jpg" alt="img"></p>
<p>不过其实上面那些操作也很难定位到具体的问题点，关键还是要看错误日志栈，找到可疑的对象，搞清楚它的回收机制，然后去分析对应的对象。<br>比如DirectByteBuffer分配内存的话，是需要full GC或者手动system.gc来进行回收的(所以最好不要使用<code>-XX:+DisableExplicitGC</code>)。</p>
<p>那么其实我们可以跟踪一下DirectByteBuffer对象的内存情况，通过<code>jmap</code>手动触发fullGC来看看堆外内存有没有被回收。</p>
<pre><code class="shell">jmap -histo:live pid</code></pre>
<ul>
<li>如果被回收了，那么大概率是堆外内存本身分配的太小了，通过<code>-XX:MaxDirectMemorySize</code>进行调整。</li>
<li>如果没有什么变化，那就要使用jmap去分析那些不能被gc的对象，以及和DirectByteBuffer之间的引用关系了。</li>
</ul>
<h2 id="4-GC问题"><a href="#4-GC问题" class="headerlink" title="4.GC问题"></a>4.GC问题</h2><p>堆内内存泄漏总是和GC异常相伴。不过GC问题不只是和内存问题相关，还有可能引起CPU负载、网络问题等系列并发症，只是相对来说和内存联系紧密些，所以我们在此单独总结一下GC相关问题。</p>
<p>我们在cpu章介绍了使用jstat来获取当前GC分代变化信息。</p>
<p>而更多时候，我们是通过GC日志来排查问题的，在启动参数中加上<code>-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps</code>来开启GC日志。</p>
<p>常见的Young GC、Full GC日志含义在此就不做赘述了。<a href="https://www.iworkh.com/book/dev/iworkh-dev-jvm/docs/#/zh-cn/3-2-jvm-gc-logger" target="_blank" rel="noopener">gc日志格式</a></p>
<p>针对gc日志，我们就能大致推断出youngGC与fullGC是否过于频繁或者耗时过长，从而对症下药。我们下面将对G1垃圾收集器来做分析，这边也建议大家使用G1<code>-XX:+UseG1GC</code>。</p>
<h3 id="4-1-youngGC过频繁"><a href="#4-1-youngGC过频繁" class="headerlink" title="4-1.youngGC过频繁"></a>4-1.youngGC过频繁</h3><p>youngGC频繁一般是短周期小对象较多，先考虑是不是Eden区/新生代设置的太小了，看能否通过调整-Xmn、-XX:SurvivorRatio等参数设置来解决问题。</p>
<p>如果参数正常，但是young gc频率还是太高，就需要使用<code>Jmap</code>和<code>MAT</code>对dump文件进行进一步排查了。</p>
<h3 id="4-2-youngGC耗时过长"><a href="#4-2-youngGC耗时过长" class="headerlink" title="4-2.youngGC耗时过长"></a>4-2.youngGC耗时过长</h3><p>耗时过长问题就要看GC日志里耗时耗在哪一块了。</p>
<p>以G1日志为例:</p>
<p>可以关注<strong>Root Scanning、Object Copy、Ref Proc</strong>等阶段。</p>
<ul>
<li><strong>Ref Proc</strong>耗时长：就要注意引用相关的对象。</li>
<li><strong>Root Scanning</strong>耗时长：就要注意线程数、跨代引用。</li>
<li><strong>Object Copy</strong>耗时长：则需要关注对象生存周期。而且耗时分析它需要横向比较，就是和其他项目或者正常时间段的耗时比较。</li>
</ul>
<p>比如说图中的Root Scanning和正常时间段比增长较多，那就是起的线程太多了。</p>
<img src="/blog/images/blog/java/analysis/2019-11-04-083826.png" width="650px">

<h3 id="4-3-触发fullGC"><a href="#4-3-触发fullGC" class="headerlink" title="4-3.触发fullGC"></a>4-3.触发fullGC</h3><p>G1中更多的还是mixedGC，但mixedGC可以和youngGC思路一样去排查。</p>
<p>触发fullGC了一般都会有问题，G1会退化使用Serial收集器来完成垃圾的清理工作，暂停时长达到秒级别，可以说是半跪了。</p>
<p>fullGC的原因可能包括以下这些，以及参数调整方面的一些思路：</p>
<ul>
<li>并发阶段失败：在并发标记阶段，MixGC之前老年代就被填满了，那么这时候G1就会放弃标记周期。这种情况，可能就需要增加堆大小，或者调整并发标记线程数<code>-XX:ConcGCThreads</code>。</li>
<li>晋升失败：在GC的时候没有足够的内存供存活/晋升对象使用，所以触发了Full GC。这时候可以通过<code>-XX:G1ReservePercent</code>来增加预留内存百分比，减少<code>-XX:InitiatingHeapOccupancyPercent</code>来提前启动标记，<code>-XX:ConcGCThreads</code>来增加标记线程数也是可以的。</li>
<li>大对象分配失败：大对象找不到合适的region空间进行分配，就会进行fullGC，这种情况下可以增大内存或者增大<code>-XX:G1HeapRegionSize</code>。</li>
<li>程序主动执行System.gc()：不要随便写就对了。</li>
</ul>
<p>另外，我们可以在启动参数中配置<code>-XX:HeapDumpPath=/xxx/dump.hprof</code>来dump fullGC相关的文件，并通过jinfo来进行gc前后的dump</p>
<pre><code class="java">jinfo -flag +HeapDumpBeforeFullGC pid 
jinfo -flag +HeapDumpAfterFullGC pid</code></pre>
<p>这样得到2份dump文件，对比后主要关注被gc掉的问题对象来定位问题。</p>
<h2 id="5-网络"><a href="#5-网络" class="headerlink" title="5.网络"></a>5.网络</h2><p>涉及到网络层面的问题一般都比较复杂，场景多，定位难，成为了大多数开发的噩梦，应该是最复杂的了。这里会举一些例子，并从<strong>tcp层、应用层</strong>以及工具的使用等方面进行阐述。</p>
<h3 id="5-1-超时"><a href="#5-1-超时" class="headerlink" title="5-1.超时"></a>5-1.超时</h3><p>超时错误大部分处在应用层面，所以这块着重理解概念。超时大体可以分为<strong>连接超时</strong>和<strong>读写超时</strong>，某些使用连接池的客户端框架还会存在获取连接超时和空闲连接清理超时。</p>
<ul>
<li>读写超时<blockquote>
<p>readTimeout/writeTimeout，有些框架叫做so_timeout或者socketTimeout，均指的是数据读写超时。注意这边的超时大部分是指逻辑上的超时。<br>soa的超时指的也是读超时。读写超时一般都只针对客户端设置。</p>
</blockquote>
</li>
<li>连接超时<blockquote>
<p>connectionTimeout，客户端通常指与服务端建立连接的最大时间。服务端这边connectionTimeout就有些五花八门了，</p>
<ul>
<li>jetty中表示空闲连接清理时间</li>
<li>tomcat则表示连接维持的最大时间。</li>
</ul>
</blockquote>
</li>
<li>其他。<blockquote>
<p>包括连接获取超时connectionAcquireTimeout和空闲连接清理超时idleConnectionTimeout。多用于使用连接池或队列的客户端或服务端框架。</p>
</blockquote>
</li>
</ul>
<p>我们在设置各种超时时间中，需要确认的是<strong>尽量保持客户端的超时 &lt; 服务端的超时</strong>，以保证连接正常结束。</p>
<p>在实际开发中，我们关心最多的应该是接口的读写超时了。</p>
<p>如何设置合理的接口超时是一个问题。如果接口超时设置的过长，那么有可能会过多地占用服务端的tcp连接。而如果接口设置的过短，那么接口超时就会非常频繁。</p>
<p>服务端接口明明rt降低，但客户端仍然一直超时又是另一个问题。这个问题其实很简单，客户端到服务端的链路包括网络传输、排队以及服务处理等，每一个环节都可能是耗时的原因。</p>
<h3 id="5-2-TCP队列溢出"><a href="#5-2-TCP队列溢出" class="headerlink" title="5-2.TCP队列溢出"></a>5-2.TCP队列溢出</h3><p>tcp队列溢出是个相对底层的错误，它可能会造成超时、rst等更表层的错误。因此错误也更隐蔽，所以我们单独说一说。</p>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083827.jpg" alt="img"></p>
<p>如上图,这里有两个队列：syns queue(半连接队列）、accept queue（全连接队列）。</p>
<p><strong>三次握手</strong></p>
<img src="/blog/images/blog/java/analysis/tcp_handshake.png" width="650px">

<ol>
<li>在server收到client的syn后，把消息放到syns queue，回复<code>syn+ack</code>给client</li>
<li>client接受到server的<code>syn+ack</code>后，发送<code>ack</code>给server</li>
<li>server收到client的ack，如果这时accept queue没满，那就从syns queue拿出暂存的信息放入accept queue中，否则按tcp_abort_on_overflow指示的执行。<ul>
<li>tcp_abort_on_overflow 0<blockquote>
<p>表示如果三次握手第三步的时候accept queue满了那么server扔掉client发过来的ack。</p>
</blockquote>
</li>
<li>tcp_abort_on_overflow 1<blockquote>
<p>则表示第三步的时候如果全连接队列满了，server发送一个rst包给client，表示废掉这个握手过程和这个连接，意味着日志里可能会有很多<code>connection reset / connection reset by peer</code>。</p>
</blockquote>
</li>
</ul>
</li>
</ol>
<p>那么在实际开发中，我们怎么能快速定位到tcp队列溢出呢？</p>
<blockquote>
<p>使用<code>netstat</code>命令</p>
</blockquote>
<pre><code class="shell">netstat -s | egrep &quot;listen|LISTEN&quot;</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-83828.jpg" alt="img"></p>
<p>如上图所示，overflowed表示全连接队列溢出的次数，sockets dropped表示半连接队列溢出的次数。</p>
<p><strong>ss命令，执行ss -lnt</strong><br><img src="/blog/images/blog/java/analysis/2019-11-04-083828.jpg" alt="img"></p>
<p>上面看到Send-Q 表示第三列的listen端口上的全连接队列最大为5，第一列Recv-Q为全连接队列当前使用了多少。</p>
<p>接着我们看看怎么设置全连接、半连接队列大小吧：</p>
<ul>
<li><strong>全连接队列</strong>的大小取决于<code>min(backlog, somaxconn)</code>。<ul>
<li>backlog是在socket创建的时候传入的</li>
<li>somaxconn是一个os级别的系统参数。</li>
</ul>
</li>
<li><strong>半连接队列</strong>的大小取决于<code>max(64, /proc/sys/net/ipv4/tcp_max_syn_backlog)</code>。</li>
</ul>
<p>在日常开发中，我们往往使用servlet容器作为服务端，所以我们有时候也需要关注容器的连接队列大小。</p>
<ul>
<li>在tomcat中backlog叫做<code>acceptCount</code></li>
<li>在jetty里面则是<code>acceptQueueSize</code>。</li>
</ul>
<h3 id="5-3-RST异常"><a href="#5-3-RST异常" class="headerlink" title="5-3.RST异常"></a>5-3.RST异常</h3><p>RST包表示连接重置，用于关闭一些无用的连接，通常表示异常关闭，区别于四次挥手。</p>
<p>在实际开发中，我们往往会看到<code>connection reset / connection reset by peer</code>错误，这种情况就是RST包导致的。</p>
<p><strong>端口不存在</strong></p>
<p>如果像不存在的端口发出建立连接SYN请求，那么服务端发现自己并没有这个端口则会直接返回一个RST报文，用于中断连接。</p>
<p><strong>主动代替FIN终止连接</strong></p>
<p>一般来说，正常的连接关闭都是需要通过FIN报文实现，然而我们也可以用RST报文来代替FIN，表示直接终止连接。实际开发中，<br>可设置SO_LINGER数值来控制，这种往往是故意的，来跳过TIMED_WAIT，提供交互效率，不闲就慎用。</p>
<p><strong>客户端或服务端有一边发生了异常，该方向对端发送RST以告知关闭连接</strong></p>
<p>我们上面讲的tcp队列溢出发送RST包其实也是属于这一种。这种往往是由于某些原因，一方无法再能正常处理请求连接了(比如程序崩了，队列满了)，从而告知另一方关闭连接。</p>
<p><strong>接收到的TCP报文不在已知的TCP连接内</strong></p>
<p>比如，一方机器由于网络实在太差TCP报文失踪了，另一方关闭了该连接，然后过了许久收到了之前失踪的TCP报文，但由于对应的TCP连接已不存在，<br>那么会直接发一个RST包以便开启新的连接。</p>
<p><strong>一方长期未收到另一方的确认报文，在一定时间或重传次数后发出RST报文</strong></p>
<p>这种大多也和网络环境相关了，网络环境差可能会导致更多的RST报文。</p>
<p>之前说过RST报文多会导致程序报错，</p>
<ul>
<li>在一个已关闭的连接上读操作会报<code>connection reset</code>，</li>
<li>在一个已关闭的连接上写操作则会报<code>connection reset by peer</code>。<br>通常我们可能还会看到<code>broken pipe</code>错误，这是管道层面的错误，表示对已关闭的管道进行读写，往往是在收到RST，<br>报出<code>connection reset</code>错后继续读写数据报的错，这个在glibc源码注释中也有介绍。</li>
</ul>
<p>我们在排查故障时候怎么确定有RST包的存在呢？</p>
<blockquote>
<p>当然是使用<code>tcpdump</code>命令进行抓包，并使用<code>wireshark</code>进行简单分析了。</p>
</blockquote>
<pre><code class="shell">tcpdump -i en0 tcp -w xxx.cap</code></pre>
<blockquote>
<p>en0表示监听的网卡。</p>
</blockquote>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083829.jpg" alt="img"></p>
<p>接下来我们通过<code>wireshark</code>打开抓到的包，可能就能看到如下图所示，红色的就表示RST包了。</p>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083830.jpg" alt="img"></p>
<h3 id="5-4-TIME-WAIT和CLOSE-WAIT"><a href="#5-4-TIME-WAIT和CLOSE-WAIT" class="headerlink" title="5-4.TIME_WAIT和CLOSE_WAIT"></a>5-4.TIME_WAIT和CLOSE_WAIT</h3><p><strong>三次握手</strong></p>
<img src="/blog/images/blog/java/analysis/tcp_handshake.png" width="650px">


<p><strong>四次挥手</strong></p>
<img src="/blog/images/blog/java/analysis/tcp_wave.png" width="650px">

<p>通过<strong>四次握手</strong>机制，TIME_WAIT和CLOSE_WAIT是啥意思相信大家都知道。</p>
<p>在线上时，我们可以直接用命令<code>netstat</code>来查看time-wait和close_wait的数量</p>
<pre><code class="shell">netstat -n | awk &#39;/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}&#39;</code></pre>
<p>用<code>ss</code>命令会更快`</p>
<pre><code class="shell">ss -ant | awk &#39;{++S[$1]} END {for(a in S) print a, S[a]}&#39;</code></pre>
<p><img src="/blog/images/blog/java/analysis/2019-11-04-083830.png" alt="img"></p>
<h4 id="5-4-1-TIME-WAIT"><a href="#5-4-1-TIME-WAIT" class="headerlink" title="5-4-1.TIME_WAIT"></a>5-4-1.TIME_WAIT</h4><p>time_wait的存在主要两个作用：</p>
<ul>
<li>一是为了丢失的数据包被后面连接复用</li>
<li>二是为了在2MSL的时间范围内正常关闭连接。<br>它的存在其实会大大减少RST包的出现。</li>
</ul>
<p>过多的time_wait在短连接频繁的场景比较容易出现。这种情况可以在服务端做一些内核参数调优:</p>
<pre><code class="shell">#表示开启重用。允许将TIME-WAIT sockets重新用于新的TCP连接，默认为0，表示关闭
net.ipv4.tcp_tw_reuse = 1
#表示开启TCP连接中TIME-WAIT sockets的快速回收，默认为0，表示关闭
net.ipv4.tcp_tw_recycle = 1</code></pre>
<p>当然我们不要忘记在NAT环境下因为时间戳错乱导致数据包被拒绝的坑了，另外的办法就是改小<code>tcp_max_tw_buckets</code>，<br>超过这个数的time_wait都会被干掉，不过这也会导致报<code>time wait bucket table overflow</code>的错。</p>
<h4 id="5-4-2-CLOSE-WAIT"><a href="#5-4-2-CLOSE-WAIT" class="headerlink" title="5-4-2.CLOSE_WAIT"></a>5-4-2.CLOSE_WAIT</h4><p>close_wait往往都是因为应用程序写的有问题，没有在ACK后再次发起FIN报文。</p>
<p>close_wait出现的概率甚至比time_wait要更高，后果也更严重。往往是由于某个地方阻塞住了，没有正常关闭连接，从而渐渐地消耗完所有的线程。</p>
<p>想要定位这类问题，最好是通过<code>jstack</code>来分析线程堆栈来排查问题，具体可参考上述章节。这里仅举一个例子。</p>
<p>开发同学说应用上线后CLOSE_WAIT就一直增多，直到挂掉为止，jstack后找到比较可疑的堆栈是大部分线程都卡在了<code>countdownlatch.await</code>方法，找开发同学了解后得知使用了多线程但是确没有catch异常，修改后发现异常仅仅是最简单的升级sdk后常出现的<code>class not found</code>。</p>

      
       <hr><span style="font-style: italic;color: gray;"> 转载请注明来源，欢迎指出任何有错误或不够清晰的表达。可以在下面评论区评论，也可以邮件至 pgmanor@qq.com </span>
    </div>
</article>


<p>
    <a  class="dashang" onclick="dashangToggle()">赏</a>
</p>


<div class="article_copyright">
    <p><span class="copy-title">文章标题:</span>JAVA线上故障排查全套路</p>
    <p><span class="copy-title">字数:</span><span class="post-count">5.7k</span></p>
    <p><span class="copy-title">本文作者:</span><a  title="pgmanor">pgmanor</a></p>
    <p><span class="copy-title">发布时间:</span>2020-12-31, 00:00:00</p>
    <p><span class="copy-title">最后更新:</span>2021-03-14, 11:27:47</p>
    <span class="copy-title">原始链接:</span><a class="post-url" href="/blog/2020/12/31/blog-java-online-analysis/" title="JAVA线上故障排查全套路">https://pgmanor.gitee.io/blog/2020/12/31/blog-java-online-analysis/</a>
    <p>
        <span class="copy-title">版权声明:</span><i class="fa fa-creative-commons"></i> <a rel="license noopener" href="http://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" title="CC BY-NC-SA 4.0 International" target = "_blank">&#34;署名-非商用-相同方式共享 4.0&#34;</a> 转载请保留原文链接及作者。
    </p>
</div>



    <div id="comments"></div>

<script src="/blog/js/valine.min.js"></script>

<script type="text/javascript">
    new Valine({
        el: '#comments',
        appId: 'g5VDso59rLvb68pq56BNnQ7A-gzGzoHsz',
        appKey: '2bebBs4aVT95Btj4j9zJve2O',
        placeholder: '说点什么...',
        avatar: 'monsterid',
        visitor : 'false'
    })
</script>




    




    </div>
    <div class="copyright">
        <p class="footer-entry">©2021 pgmanor</p>
<!--<p class="footer-entry">Built with <a href="https://hexo.io/" target="_blank">Hexo</a> and <a href="https://github.com/yelog/hexo-theme-3-hexo" target="_blank">3-hexo</a> theme</p>-->

    </div>
    <div class="full-toc">
        <button class="full" data-title="切换全屏 快捷键 s"><span class="min "></span></button>
<button class="post-toc-menu" data-title="打开TOC 快捷键 w"><span class="post-toc-menu-icons"></span></button>
<div class="post-toc"><span class="post-toc-title">目录</span>
    <div class="post-toc-content">

    </div>
</div>
<a class="" id="rocket" ></a>

    </div>
</div>
<div class="acParent"></div>

<div class="hide_box" onclick="dashangToggle()"></div>
<div class="shang_box">
    <a class="shang_close"  onclick="dashangToggle()">×</a>
    <div class="shang_tit">
        <p>喜欢就点赞,疼爱就打赏</p>
    </div>
    <div class="shang_payimg">
        <div class="pay_img">
            <img src="/blog/img/alipay.jpg" class="alipay" title="扫码支持">
            <img src="/blog/img/weixin.jpg" class="weixin" title="扫码支持">
        </div>
    </div>
    <div class="shang_payselect">
        <span><label><input type="radio" name="pay" checked value="alipay">支付宝</label></span><span><label><input type="radio" name="pay" value="weixin">微信</label></span>
    </div>
</div>


</body>
<script src="/blog/js/jquery.pjax.js?v=1.0.1x" ></script>

<script src="/blog/js/script.js?v=1.0.1x" ></script>
<script>
    var img_resize = 'default';
    /*作者、标签的自动补全*/
    $(function () {
        $('.search').AutoComplete({
            'data': ['@pgmanor','#AQS','#attr','#aysnc','#beanmap','#bigdecimal','#blog','#chrome','#dict','#dll','#doc','#docker','#domain','#elementadmin','#es','#excel','#farm','#fastjson','#func','#git','#github','#guide','#hadoop','#hexo','#idea','#initialzr','#jackson','#java','#jboss','#jdni','#jetbrains','#jfinalshell','#joplin','#JPA','#junit','#jupyter','#keepavlie','#kill','#linux','#LocalDate','#lock','#markdown','#maven','#message','#mock','#mq','#mycat','#mysql','#nginx','#notebook','#postman','#powermock','#python','#redis','#rest','#sharding-jdbc','#spark','#springboot','#testng','#tornado','#track','#transactional','#uuid','#vuepress','#web','#windows','#zookeeper','#分库分表','#分页','#加密','#启动','#工具','#并发','#序列化','#微服务','#性能分析','#手册','#有道云','#模板','#百度云盘','#算法','#面试','#项目',],
            'itemHeight': 20,
            'width': 418
        }).AutoComplete('show');
    })
    function initArticle() {
        /*渲染对应的表格样式*/
        
            $(".post .pjax table").addClass("green_title");
        

        /*渲染打赏样式*/
        
        $("input[name=pay]").on("click", function () {
            if($("input[name=pay]:checked").val()=="weixin"){
                $(".shang_box .shang_payimg .pay_img").addClass("weixin_img");
            } else {
                $(".shang_box .shang_payimg .pay_img").removeClass("weixin_img");
            }
        })
        

        /*高亮代码块行号*/
        
        $('pre code').each(function(){
            var lines = $(this).text().trim().split('\n').length, widther='';
            if (lines>99) {
                widther = 'widther'
            }
            var $numbering = $('<ul/>').addClass('pre-numbering ' + widther).attr("unselectable","on");
            $(this).addClass('has-numbering ' + widther)
                    .parent()
                    .append($numbering);
            for(var i=1;i<=lines;i++){
                $numbering.append($('<li/>').text(i));
            }
        });
        

        /*访问数量*/
        
        $.getScript("//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js");
        

        /*代码高亮，行号对齐*/
        $('.pre-numbering').css('line-height',$('.has-numbering').css('line-height'));

        
        
    }

    /*打赏页面隐藏与展示*/
    
    function dashangToggle() {
        $(".shang_box").fadeToggle();
        $(".hide_box").fadeToggle();
    }
    

</script>

<!--加入行号的高亮代码块样式-->

<style>
    pre{
        position: relative;
        margin-bottom: 24px;
        border-radius: 10px;
        border: 1px solid #e2dede;
        background: #FFF;
        overflow: hidden;
    }
    code.has-numbering{
        margin-left: 30px;
    }
    code.has-numbering.widther{
        margin-left: 35px;
    }
    .pre-numbering{
        margin: 0px;
        position: absolute;
        top: 0;
        left: 0;
        width: 20px;
        padding: 0.5em 3px 0.7em 5px;
        border-right: 1px solid #C3CCD0;
        text-align: right;
        color: #AAA;
        background-color: #fafafa;
    }
    .pre-numbering.widther {
        width: 35px;
    }
</style>

<!--自定义样式设置-->
<style>
    
    
    .nav {
        width: 442px;
    }
    .nav.fullscreen {
        margin-left: -442px;
    }
    .nav-left {
        width: 120px;
    }
    
    
    @media screen and (max-width: 1468px) {
        .nav {
            width: 442px;
        }
        .nav.fullscreen {
            margin-left: -442px;
        }
        .nav-left {
            width: 150px;
        }
    }
    
    
    @media screen and (max-width: 1024px) {
        .nav {
            width: 442px;
            margin-left: -442px
        }
        .nav.fullscreen {
            margin-left: 0;
        }
        .nav .hide-list.fullscreen {
            left: 442px
        }
    }
    
    @media screen and (max-width: 426px) {
        .nav {
            width: 100%;
        }
        .nav-left {
            width: 100%;
        }
    }
    
    
    .nav-right .title-list nav a .post-title, .nav-right .title-list #local-search-result a .post-title {
        color: #383636;
    }
    
    
    .nav-right .title-list nav a .post-date, .nav-right .title-list #local-search-result a .post-date {
        color: #5e5e5f;
    }
    
    
    .nav-right nav a.hover, #local-search-result a.hover{
        background-color: #e2e0e0;
    }
    
    

    /*列表样式*/
    
    .post .pjax article .article-entry>ol, .post .pjax article .article-entry>ul, .post .pjax article>ol, .post .pjax article>ul{
        border: #e2dede solid 1px;
        border-radius: 10px;
        padding: 10px 32px 10px 56px;
    }
    .post .pjax article .article-entry li>ol, .post .pjax article .article-entry li>ul,.post .pjax article li>ol, .post .pjax article li>ul{
        padding-top: 5px;
        padding-bottom: 5px;
    }
    .post .pjax article .article-entry>ol>li, .post .pjax article .article-entry>ul>li,.post .pjax article>ol>li, .post .pjax article>ul>li{
        margin-bottom: auto;
        margin-left: auto;
    }
    .post .pjax article .article-entry li>ol>li, .post .pjax article .article-entry li>ul>li,.post .pjax article li>ol>li, .post .pjax article li>ul>li{
        margin-bottom: auto;
        margin-left: auto;
    }
    

    /* 背景图样式 */
    
    


    /*引用块样式*/
    

    /*文章列表背景图*/
    

    
</style>






<div class="mobile-menus-out" >

</div>
<div class="mobile-menus">
    
    
    
    <a class="dynamic-menu " target="_blank"  href="https://www.iworkh.com/">iworkh</a>
    
    
    <a class="dynamic-menu " target="_blank"  href="https://gitee.com/pgmanor">gitee</a>
    
</div>


<div style="position:absolute; bottom: 0; right: 0;">
    <iframe frameborder="no" border="0" marginwidth="0" marginheight="0" width=150 height=40 src="//music.163.com/outchain/player?type=2&id=1323150930&auto=0&height=32"></iframe>
</div>

</html>
